{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import csv\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import gzip\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_DATA = 'Data/'\n",
    "PATH_TO_DATA_EN = PATH_TO_DATA+\"enwiki/\"\n",
    "PATH_TO_DATA_UK = PATH_TO_DATA+\"ukwiki/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "EN_ID_NAME = \"enwiki-20180620-id_name.csv.gz\"\n",
    "EN_LINKS = \"enwiki-20180620-pages-links.csv.gz\"\n",
    "EN_ALIASES = \"enwiki-20180620-id_alias_title_alias.csv.gz\"\n",
    "EN_ALIASES_TO_ORIG = \"enwiki-20180620-id_alias_id_orig.csv.gz\"\n",
    "EN_ORIG_ALIASES = \"enwiki-20180620-id_orig_title_alias.csv.gz\"\n",
    "EN_RED_NAME_COUNT = \"enwiki-20180620-red_name_count.csv.gz\"\n",
    "EN_RED_COUNT_COUNT = \"enwiki-20180620-red_count_by_count.csv.gz\"\n",
    "\n",
    "UK_ID_NAME = \"ukwiki-20180620-id_name.csv.gz\"\n",
    "UK_LINKS = \"ukwiki-20180620-pages-links.csv.gz\"\n",
    "UK_ALIASES = \"ukwiki-20180620-id_alias_title_alias.csv.gz\"\n",
    "UK_ALIASES_TO_ORIG = \"ukwiki-20180620-id_alias_id_orig.csv.gz\"\n",
    "UK_ORIG_ALIASES = \"ukwiki-20180620-id_orig_title_alias.csv.gz\"\n",
    "UK_RED_NAME_COUNT = \"ukwiki-20180620-red_name_count.csv.gz\"\n",
    "UK_RED_COUNT_COUNT = \"ukwiki-20180620-red_count_by_count.csv.gz\"\n",
    "\n",
    "UK_EN_LINKS = \"20180620-langlinks_uk_en.csv.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpack(file_name):\n",
    "    file_name_new = file_name.replace(\".gz\",\"\")\n",
    "    with gzip.open(file_name, 'rb') as f_in, open(file_name_new, 'wb') as f_out:\n",
    "        f_out.writelines(f_in)\n",
    "    return file_name_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pack_and_remove(file_name):\n",
    "    file_name_new = file_name+'.gz'\n",
    "    with open(file_name, 'rb') as f_in, gzip.open(file_name_new, 'wb') as f_out:\n",
    "        f_out.writelines(f_in)\n",
    "    os.remove(file_name)\n",
    "    return file_name_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-pages-links.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>link_id</th>\n",
       "      <th>link_pos</th>\n",
       "      <th>link_pos_perc</th>\n",
       "      <th>link_val</th>\n",
       "      <th>link_text</th>\n",
       "      <th>is_red_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>1564700</td>\n",
       "      <td>12165</td>\n",
       "      <td>0</td>\n",
       "      <td>Online etymology dictionary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>142615</td>\n",
       "      <td>13201</td>\n",
       "      <td>0</td>\n",
       "      <td>William Godwin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>44379723</td>\n",
       "      <td>13224</td>\n",
       "      <td>0</td>\n",
       "      <td>Wilhelm Weitling</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>1389427</td>\n",
       "      <td>14539</td>\n",
       "      <td>0</td>\n",
       "      <td>Foundation for Economic Education</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>49873055</td>\n",
       "      <td>14796</td>\n",
       "      <td>0</td>\n",
       "      <td>Anarchist Seeds Beneath the Snow</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id   link_id  link_pos  link_pos_perc                           link_val  \\\n",
       "0  12   1564700     12165              0        Online etymology dictionary   \n",
       "1  12    142615     13201              0                     William Godwin   \n",
       "2  12  44379723     13224              0                   Wilhelm Weitling   \n",
       "3  12   1389427     14539              0  Foundation for Economic Education   \n",
       "4  12  49873055     14796              0   Anarchist Seeds Beneath the Snow   \n",
       "\n",
       "  link_text  is_red_link  \n",
       "0       NaN        False  \n",
       "1       NaN        False  \n",
       "2       NaN        False  \n",
       "3       NaN        False  \n",
       "4       NaN        False  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_LINKS\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_articles = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_articles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-id_name.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>Anarchism</td>\n",
       "      <td>187065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>Autism</td>\n",
       "      <td>154793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>39</td>\n",
       "      <td>Albedo</td>\n",
       "      <td>39569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>290</td>\n",
       "      <td>A</td>\n",
       "      <td>24741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>303</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>184832</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id      title  length\n",
       "0   12  Anarchism  187065\n",
       "1   25     Autism  154793\n",
       "2   39     Albedo   39569\n",
       "3  290          A   24741\n",
       "4  303    Alabama  184832"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_ID_NAME\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_id_name = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", escapechar =\"\\\\\")\n",
    "os.remove(fn_new)\n",
    "df_en_id_name.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-id_alias_title_alias.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_alias</th>\n",
       "      <th>title_alias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>AccessibleComputing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>AfghanistanHistory</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>AfghanistanGeography</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>AfghanistanPeople</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>AfghanistanCommunications</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_alias                title_alias\n",
       "0        10        AccessibleComputing\n",
       "1        13         AfghanistanHistory\n",
       "2        14       AfghanistanGeography\n",
       "3        15          AfghanistanPeople\n",
       "4        18  AfghanistanCommunications"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_ALIASES\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_id_alias_title_alias = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", escapechar =\"\\\\\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_id_alias_title_alias.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-id_alias_id_orig.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_alias</th>\n",
       "      <th>id_orig</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>411964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>13813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>12681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>66468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>6684</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_alias  id_orig\n",
       "0        10   411964\n",
       "1        13    13813\n",
       "2        14    12681\n",
       "3        15    66468\n",
       "4        18     6684"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_ALIASES_TO_ORIG\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_id_alias_id_orig = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", escapechar =\"\\\\\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_id_alias_id_orig.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-id_orig_title_alias.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_orig</th>\n",
       "      <th>title_alias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>411964</td>\n",
       "      <td>AccessibleComputing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13813</td>\n",
       "      <td>AfghanistanHistory</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12681</td>\n",
       "      <td>AfghanistanGeography</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>66468</td>\n",
       "      <td>AfghanistanPeople</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6684</td>\n",
       "      <td>AfghanistanCommunications</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_orig                title_alias\n",
       "0   411964        AccessibleComputing\n",
       "1    13813         AfghanistanHistory\n",
       "2    12681       AfghanistanGeography\n",
       "3    66468          AfghanistanPeople\n",
       "4     6684  AfghanistanCommunications"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_ORIG_ALIASES\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_id_orig_title_alias = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", escapechar =\"\\\\\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_id_orig_title_alias.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-red_name_count.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link_title</th>\n",
       "      <th>in_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>List of South Korean films of 19xx</td>\n",
       "      <td>4451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Poll</td>\n",
       "      <td>1547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Philippine general election,</td>\n",
       "      <td>1463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>List of Quebec provincial electoral districts</td>\n",
       "      <td>1295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Saaremaa Parish</td>\n",
       "      <td>841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      link_title  in_count\n",
       "0             List of South Korean films of 19xx      4451\n",
       "1                                       The Poll      1547\n",
       "2                  Philippine general election,       1463\n",
       "3  List of Quebec provincial electoral districts      1295\n",
       "4                                Saaremaa Parish       841"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_RED_NAME_COUNT\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_red_name_count_in = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_red_name_count_in.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/enwiki/enwiki-20180620-red_count_by_count.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>in_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>4354094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>811612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>267891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>129747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>69636</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   count  in_count\n",
       "0      1   4354094\n",
       "1      2    811612\n",
       "2      3    267891\n",
       "3      4    129747\n",
       "4      5     69636"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_EN+EN_RED_COUNT_COUNT\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_en_red_count_count_in = pd.read_csv(fn_new, encoding='ISO-8859-1', nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_en_red_count_count_in.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-pages-links.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>link_id</th>\n",
       "      <th>link_pos</th>\n",
       "      <th>link_pos_perc</th>\n",
       "      <th>link_val</th>\n",
       "      <th>link_text</th>\n",
       "      <th>is_red_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>884280</td>\n",
       "      <td>1228</td>\n",
       "      <td>0</td>\n",
       "      <td>Об'єкт (філософія)</td>\n",
       "      <td>Об'єкт</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>21623</td>\n",
       "      <td>1533</td>\n",
       "      <td>0</td>\n",
       "      <td>Ератосфен</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>274944</td>\n",
       "      <td>1599</td>\n",
       "      <td>0</td>\n",
       "      <td>географ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>26194</td>\n",
       "      <td>1611</td>\n",
       "      <td>0</td>\n",
       "      <td>Клавдій Птолемей</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>20459</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>Відродження</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  link_id  link_pos  link_pos_perc            link_val link_text  \\\n",
       "0  13   884280      1228              0  Об'єкт (філософія)    Об'єкт   \n",
       "1  13    21623      1533              0           Ератосфен       NaN   \n",
       "2  13   274944      1599              0             географ       NaN   \n",
       "3  13    26194      1611              0    Клавдій Птолемей       NaN   \n",
       "4  13    20459      1722              0         Відродження       NaN   \n",
       "\n",
       "   is_red_link  \n",
       "0        False  \n",
       "1        False  \n",
       "2        False  \n",
       "3        False  \n",
       "4        False  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_LINKS\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_articles = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20) \n",
    "os.remove(fn_new)\n",
    "df_uk_articles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-id_name.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Головна сторінка</td>\n",
       "      <td>3197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>Географія</td>\n",
       "      <td>16843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>584</td>\n",
       "      <td>Атом</td>\n",
       "      <td>35112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>585</td>\n",
       "      <td>Мільярд</td>\n",
       "      <td>1759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>586</td>\n",
       "      <td>Ядро</td>\n",
       "      <td>8064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id             title  length\n",
       "0    3  Головна сторінка    3197\n",
       "1   13         Географія   16843\n",
       "2  584              Атом   35112\n",
       "3  585           Мільярд    1759\n",
       "4  586              Ядро    8064"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_ID_NAME\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_id_name = pd.read_csv(fn_new,  encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_id_name.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-id_alias_title_alias.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_alias</th>\n",
       "      <th>title_alias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>Esperanto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>Wikipedia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>HomePage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>647</td>\n",
       "      <td>Капустянко Микола</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>693</td>\n",
       "      <td>Володимир Винниченко</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_alias           title_alias\n",
       "0         2             Esperanto\n",
       "1         6             Wikipedia\n",
       "2         9              HomePage\n",
       "3       647     Капустянко Микола\n",
       "4       693  Володимир Винниченко"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_ALIASES\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_id_alias_title_alias = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_id_alias_title_alias.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-id_alias_id_orig.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_alias</th>\n",
       "      <th>id_orig</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>9701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>647</td>\n",
       "      <td>61817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>693</td>\n",
       "      <td>6083</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_alias  id_orig\n",
       "0         2      610\n",
       "1         6     9701\n",
       "2         9        3\n",
       "3       647    61817\n",
       "4       693     6083"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_ALIASES_TO_ORIG\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_id_alias_id_orig = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_id_alias_id_orig.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-id_orig_title_alias.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_orig</th>\n",
       "      <th>title_alias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>610</td>\n",
       "      <td>Esperanto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9701</td>\n",
       "      <td>Wikipedia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>HomePage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61817</td>\n",
       "      <td>Капустянко Микола</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6083</td>\n",
       "      <td>Володимир Винниченко</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_orig           title_alias\n",
       "0      610             Esperanto\n",
       "1     9701             Wikipedia\n",
       "2        3              HomePage\n",
       "3    61817     Капустянко Микола\n",
       "4     6083  Володимир Винниченко"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_ORIG_ALIASES\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_id_orig_title_alias = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_id_orig_title_alias.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-red_name_count.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link_title</th>\n",
       "      <th>in_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(сузір'я)</td>\n",
       "      <td>15716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Сеніор</td>\n",
       "      <td>1660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ацетилювання</td>\n",
       "      <td>1619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Подравський регіон‎</td>\n",
       "      <td>1298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>білок розвитку</td>\n",
       "      <td>942</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            link_title  in_count\n",
       "0            (сузір'я)     15716\n",
       "1               Сеніор      1660\n",
       "2         ацетилювання      1619\n",
       "3  Подравський регіон‎      1298\n",
       "4       білок розвитку       942"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_RED_NAME_COUNT\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_red_name_count_in = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_red_name_count_in.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/ukwiki/ukwiki-20180620-red_count_by_count.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>in_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1177094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>203920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>68509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>33237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>18475</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   count  in_count\n",
       "0      1   1177094\n",
       "1      2    203920\n",
       "2      3     68509\n",
       "3      4     33237\n",
       "4      5     18475"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA_UK+UK_RED_COUNT_COUNT\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_red_name_count_in = pd.read_csv(fn_new, encoding='UTF-8', quotechar=\"\\\"\", nrows=20)\n",
    "os.remove(fn_new)\n",
    "df_uk_red_name_count_in.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data/20180620-langlinks_uk_en.csv.gz\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_uk</th>\n",
       "      <th>id_en</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>49244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93215</td>\n",
       "      <td>49244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>109484</td>\n",
       "      <td>49244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>122051</td>\n",
       "      <td>49244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>130506</td>\n",
       "      <td>49244</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id_uk  id_en\n",
       "0       3  49244\n",
       "1   93215  49244\n",
       "2  109484  49244\n",
       "3  122051  49244\n",
       "4  130506  49244"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = PATH_TO_DATA+UK_EN_LINKS\n",
    "print(fn)\n",
    "fn_new = unpack(fn)\n",
    "df_uk_en_links = pd.read_csv(fn_new, encoding='ISO-8859-1', quotechar=\"'\")\n",
    "os.remove(fn_new)\n",
    "df_uk_en_links.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
